{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对用户进行聚类\n",
    "\n",
    "数据来源于Kaggle竞赛：Event Recommendation Engine Challenge，根据\n",
    "events they’ve responded to in the past\n",
    "user demographic information\n",
    "what events they’ve seen and clicked on in our app\n",
    "用户对某个事件是否感兴趣\n",
    "\n",
    "竞赛官网：\n",
    "https://www.kaggle.com/c/event-recommendation-engine-challenge/data\n",
    "\n",
    "由于用户众多（3w+），可以对用户进行聚类\n",
    "事件描述信息在users.csv文件：共110维特征\n",
    "user_id\n",
    "locale：地区，语言\n",
    "birthyear：出身年\n",
    "gender：性别\n",
    "joinedAt：用户加入APP的时间，ISO-8601 UTC time\n",
    "location：地点\n",
    "timezone：时区\n",
    "\n",
    "作业要求：\n",
    "根据用户的属性进行聚类（KMeans聚类）\n",
    "尝试K=20， 40， 80，并计算各自CH_scores。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 导入工具包\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import hashlib\n",
    "import locale\n",
    "\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>location</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3197468391</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3537982273</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>823183725</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>Stratford  Ontario</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1872223848</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>Tehran  Iran</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3429017717</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id locale birthyear  gender                  joinedAt  \\\n",
       "0  3197468391  id_ID      1993    male  2012-10-02T06:40:55.524Z   \n",
       "1  3537982273  id_ID      1992    male  2012-09-29T18:03:12.111Z   \n",
       "2   823183725  en_US      1975    male  2012-10-06T03:14:07.149Z   \n",
       "3  1872223848  en_US      1991  female  2012-11-04T08:59:43.783Z   \n",
       "4  3429017717  id_ID      1995  female  2012-09-10T16:06:53.132Z   \n",
       "\n",
       "             location  timezone  \n",
       "0    Medan  Indonesia     480.0  \n",
       "1    Medan  Indonesia     420.0  \n",
       "2  Stratford  Ontario    -240.0  \n",
       "3        Tehran  Iran     210.0  \n",
       "4                 NaN     420.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据\n",
    "import pandas as pd\n",
    "df = pd.read_csv(\"users.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#丢弃user_id\n",
    "X_train = df.drop('user_id',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>location</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>Stratford  Ontario</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>Tehran  Iran</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  locale birthyear  gender                  joinedAt            location  \\\n",
       "0  id_ID      1993    male  2012-10-02T06:40:55.524Z    Medan  Indonesia   \n",
       "1  id_ID      1992    male  2012-09-29T18:03:12.111Z    Medan  Indonesia   \n",
       "2  en_US      1975    male  2012-10-06T03:14:07.149Z  Stratford  Ontario   \n",
       "3  en_US      1991  female  2012-11-04T08:59:43.783Z        Tehran  Iran   \n",
       "4  id_ID      1995  female  2012-09-10T16:06:53.132Z                 NaN   \n",
       "\n",
       "   timezone  \n",
       "0     480.0  \n",
       "1     420.0  \n",
       "2    -240.0  \n",
       "3     210.0  \n",
       "4     420.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['id_ID', 'en_US', 'ka_GE', 'es_LA', 'fr_FR', 'ar_AR', 'en_GB',\n",
       "       'pt_BR', 'th_TH', 'vi_VN', 'fa_IR', 'es_ES', 'hu_HU', 'cs_CZ',\n",
       "       'pt_PT', 'bs_BA', 'ko_KR', 'ru_RU', 'zh_TW', 'de_DE', 'mn_MN',\n",
       "       'zh_CN', 'ja_JP', 'bg_BG', 'km_KH', 'it_IT', 'fr_CA', 'fi_FI',\n",
       "       'tr_TR', 'sq_AL', 'zh_HK', 'el_GR', 'jv_ID', 'ro_RO', 'sk_SK',\n",
       "       'en_PI', 'pl_PL', 'ms_MY', 'az_AZ', 'nl_NL', 'hr_HR', 'en_IN',\n",
       "       'hi_IN', 'ca_ES', 'da_DK', 'fb_LT', 'uk_UA', 'nb_NO', 'sv_SE',\n",
       "       'et_EE', 'bn_IN', 'sr_RS', 'lt_LT', 'pa_IN', 'af_ZA', 'lv_LV',\n",
       "       'en_UD', 'tl_PH', 'eo_EO', 'he_IL', 'mk_MK', 'ku_TR', 'es_MX',\n",
       "       'cy_GB'], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train['locale'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "查看各个属性的空缺值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 38209 entries, 0 to 38208\n",
      "Data columns (total 6 columns):\n",
      "locale       38209 non-null object\n",
      "birthyear    38209 non-null object\n",
      "gender       38100 non-null object\n",
      "joinedAt     38152 non-null object\n",
      "location     32745 non-null object\n",
      "timezone     37773 non-null float64\n",
      "dtypes: float64(1), object(5)\n",
      "memory usage: 1.7+ MB\n"
     ]
    }
   ],
   "source": [
    "X_train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对 locale进行 LabelEncoder编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "X_train['locale']= le.fit_transform(X_train['locale'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([31, 16, 35, 19, 26,  1, 12, 48, 56, 60, 22, 18, 30,  7, 49,  5, 37,\n",
       "       51, 63, 10, 42, 61, 33,  3, 36, 32, 25, 24, 58, 53, 62, 11, 34, 50,\n",
       "       52, 14, 47, 43,  2, 45, 29, 13, 28,  6,  9, 23, 59, 44, 55, 21,  4,\n",
       "       54, 39, 46,  0, 40, 15, 57, 17, 27, 41, 38, 20,  8], dtype=int64)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train['locale'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38209"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 根据birthyear计算年龄"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train['age']=np.zeros(X_train.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Getage(birthyear):\n",
    "    try:\n",
    "        if(birthyear == 'None'):\n",
    "            year = 1990\n",
    "        else:\n",
    "            year = birthyear\n",
    "        return 2018-int(year)\n",
    "    except:\n",
    "        return (2018- 1990) #1990是均值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0        25.0\n",
      "1        26.0\n",
      "2        43.0\n",
      "3        27.0\n",
      "4        23.0\n",
      "5        45.0\n",
      "6        24.0\n",
      "7        53.0\n",
      "8        39.0\n",
      "9        30.0\n",
      "10       26.0\n",
      "11       22.0\n",
      "12       25.0\n",
      "13       22.0\n",
      "14       28.0\n",
      "15       26.0\n",
      "16       25.0\n",
      "17       28.0\n",
      "18       25.0\n",
      "19       28.0\n",
      "20       30.0\n",
      "21       30.0\n",
      "22       26.0\n",
      "23       24.0\n",
      "24       24.0\n",
      "25       31.0\n",
      "26       25.0\n",
      "27       30.0\n",
      "28       24.0\n",
      "29       30.0\n",
      "         ... \n",
      "38179    26.0\n",
      "38180    23.0\n",
      "38181    33.0\n",
      "38182    35.0\n",
      "38183    28.0\n",
      "38184    22.0\n",
      "38185    24.0\n",
      "38186    24.0\n",
      "38187    29.0\n",
      "38188    31.0\n",
      "38189    61.0\n",
      "38190    30.0\n",
      "38191    22.0\n",
      "38192    20.0\n",
      "38193    38.0\n",
      "38194    23.0\n",
      "38195    34.0\n",
      "38196    26.0\n",
      "38197    24.0\n",
      "38198    27.0\n",
      "38199    23.0\n",
      "38200    29.0\n",
      "38201    21.0\n",
      "38202    21.0\n",
      "38203    25.0\n",
      "38204    52.0\n",
      "38205    21.0\n",
      "38206    23.0\n",
      "38207    29.0\n",
      "38208    38.0\n",
      "Name: age, Length: 38209, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print( X_train['age'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对性别进行编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "genderIdMap = {\"NaN\":0, \"male\":1, \"female\":2}\n",
    "X_train['gender'].fillna('NaN',inplace=True)\n",
    "X_train['gender'] = X_train['gender'].apply(lambda x :genderIdMap[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对joinedAt时间进行编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getJoinedYearMonth(dateString):\n",
    "    try:\n",
    "        dttm = datetime.datetime.strptime(dateString,\"%Y-%m-%dT%H:%M:%S.%fZ\") \n",
    "        return (dttm.year-2010)*12 + dttm.month        \n",
    "    except:  #缺失补0\n",
    "         return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 由于location缺失太多，抛弃location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.drop('location',axis =1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对timezone进行编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    " def getTimezoneInt(timezone):\n",
    "        try:\n",
    "          return int(timezone)\n",
    "        except:  #缺失值处理\n",
    "          return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "userMatrix = np.zeros((X_train.shape[0],5), dtype=np.int)\n",
    "\n",
    "for i in np.arange(0,X_train.shape[0]):\n",
    "    userMatrix[i,0] = X_train.loc[i,'locale']\n",
    "    userMatrix[i,1] = Getage(X_train.loc[i,'age'])\n",
    "    userMatrix[i,2] = X_train.loc[i,'gender']\n",
    "    userMatrix[i,3] = getJoinedYearMonth(X_train.loc[i,'joinedAt'])\n",
    "    userMatrix[i,4] = getTimezoneInt(X_train.loc[i,'timezone'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对userMatrix进行归一化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "userMatrix = normalize(userMatrix, norm=\"l1\", axis=0, copy=False)\n",
    "cols = ['LocaleId', 'age', 'GenderId', 'JoinedYearMonth', 'TimezoneInt']\n",
    "FE_data = pd.DataFrame(data=userMatrix, columns=cols)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LocaleId</th>\n",
       "      <th>age</th>\n",
       "      <th>GenderId</th>\n",
       "      <th>JoinedYearMonth</th>\n",
       "      <th>TimezoneInt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>-0.000018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000031</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   LocaleId       age  GenderId  JoinedYearMonth  TimezoneInt\n",
       "0  0.000036  0.000026  0.000019         0.000026     0.000036\n",
       "1  0.000036  0.000026  0.000019         0.000026     0.000031\n",
       "2  0.000019  0.000026  0.000019         0.000026    -0.000018\n",
       "3  0.000019  0.000026  0.000038         0.000027     0.000016\n",
       "4  0.000036  0.000026  0.000038         0.000026     0.000031"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FE_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 存入CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "FE_RESULT = pd.DataFrame(FE_data,columns=cols)\n",
    "FE_RESULT.to_csv('FE_ym.csv',header=True,index_label='id')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
